import json, os
import requests
from pyquery import PyQuery as pq
import logging
import random

_path = os.path.dirname(__file__)
logger = logging.getLogger()

logging.basicConfig(level=logging.INFO, filename=os.path.join(_path, 'log/get_ip_log.txt'))


class Get_IP(object):
    """
        从免费的ip 代理网站上抓一些免费的ip地址构成字典,写到json文件中
    """
    def __init__(self, url, result_ip_file):
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'}
        self.request_s = requests.session()
        self.url = url
        self.result_ip_file = result_ip_file
        self.list_date = list()

    def write_get_ip_file(self):
        with open(self.result_ip_file, 'w', encoding='utf8') as f:
            json.dump(self.list_date, fp=f, ensure_ascii=False, indent=2)

    def get_ip_info(self, trs):
        # 提取每个 tr 的数据
        for tr in trs:
            ip = tr('td').eq(1).text().strip()   # ip
            port = tr('td').eq(2).text().strip()  # port
            type = tr('td').eq(5).text().strip()

            # if tr.attr('class') == 'subtitle':
            #     logging.info('have class subtitle ---title')  # 标题，不要
            #     continue
            # if tr('a').text() == '更多':
            #     logging.info('have a more---title')  # 标题，不要
            #     continue
            if tr('th'):
                logging.info('page  th ---title')  # 标题，不要
                continue

            ip_live_time = tr('td').eq(6).text()
            if '分钟' in ip_live_time:
                logging.info('ip [{}] live too short'.format(ip))  # ip存活太短，不要
                continue

            if 'socks' in type:
                logging.info('ip [{}] is socks'.format(ip))  # 不是http和https协议，不要
                continue

            ip_dict = {  # 构造字典
                '{}'.format(type): type + '://' + ip + port
            }
            logging.info('ok ip---->{}'.format(ip_dict))

            self.list_date.append(ip_dict)   # 添加到全局的列表中

    def get_response(self):

        response = self.request_s.get(self.url, headers=self.headers, timeout=60).content.decode()
        response = pq(response)
        trs = response('table>tr').items()
        self.get_ip_info(trs=trs)

        # response = self.read_file(file='IP-nn.html')
        for i in range(10):  # 国内高匿代理
            response = self.request_s.get(self.url+'nn/{}'.format(i), headers=self.headers, timeout=60).content.decode()
            response = pq(response)
            trs = response('table>tr').items()
            self.get_ip_info(trs=trs)

        for j in range(10):  # 国内普通代理
            response = self.request_s.get(self.url+'nt/{}'.format(j), headers=self.headers, timeout=60).content.decode()
            response = pq(response)
            trs = response('table>tr').items()
            self.get_ip_info(trs=trs)

        for n in range(10):  # 国内HTTPS代理
            response = self.request_s.get(self.url+'wn/{}'.format(n), headers=self.headers, timeout=60).content.decode()
            response = pq(response)
            trs = response('table>tr').items()
            self.get_ip_info(trs=trs)

        for m in range(10):  # 国内HTTP代理
            response = self.request_s.get(self.url+'wt/{}'.format(m), headers=self.headers, timeout=60).content.decode()
            response = pq(response)
            trs = response('table>tr').items()
            self.get_ip_info(trs=trs)

    def main(self):
        self.get_response()
        self.write_get_ip_file()


if __name__ == '__main__':
    result_ip_file = os.path.join(_path, 'proxy_ip/get_ip_data.json')
    get_ip = Get_IP(url='https://www.xicidaili.com/', result_ip_file=result_ip_file)
    get_ip.main()
